package com.onpositive.wikipedialookup;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashSet;


public class WikipediaRefsLookup extends AbstractRefsLookUp {
	
	private String storageFilePath = "C:/wikistorage/wiki refs/enwiki-20111007-all-titles-in-ns0" ;
	private String commonWordsFilePath = "C:/wikistorage/wiki refs/common words.txt" ;
	
	public static WikipediaRefsLookup getInstance(){
		
		if( instance == null ){
			instance = new WikipediaRefsLookup() ;
			instance.init() ;			
		}
		
		return instance ;		
	}
	
	public static void disposeInstance(){
		
		if( instance != null )
			instance.dispose() ;
	}
	
	private static WikipediaRefsLookup instance ;
	
	private WikipediaRefsLookup(){}
	
	private void init()
	{
		linksMap = new RefMap( 10000000 ) ;
		File f  = new File(storageFilePath) ;
		File f0 = new File(commonWordsFilePath) ;
		int number = 0 ;
	
		try {
			FileReader freader = new FileReader(f) ;
			BufferedReader br = new BufferedReader( freader )  ;			
			
			boolean mayContinue = true ;
			while( mayContinue )
			{
				System.out.print( "" + number + "\n" ) ;
				for( int i = 0 ; i < 500000 ;i++ )
				{
					String line = br.readLine();
					if (line == null ){
						mayContinue = false ;
						break;
					}
					
					int count = 0 ;
					for( int ind = line.indexOf('_') ; ind > 0 ; ind = line.indexOf( '_', ind+1 ) )
						count++ ;
					
					maxCount = count > maxCount ? count : maxCount ;
					linksMap.add( line.toLowerCase() ) ;
				}
				number += 500000 ;
			}
			
			freader = new FileReader(f0) ;
			br = new BufferedReader( freader )  ;			
			
			while( true )
			{
				String line = br.readLine() ;
				if (line == null )
					break;
					
				line = line.trim() ;
				if( line.length() == 0 )
					continue ;
				
				int start = 0 ;
				for( ; Character.isDigit( line.charAt(start) ) ; start++ ) ;
				line = line.substring(start).trim() ;
				if( Character.isUpperCase( line.charAt(0) ) )
					continue ;
				
				if( linksMap.remove(line.toLowerCase()) != null )
				{
					System.out.print("removed: ") ;
					System.out.print(line) ;
					System.out.print("\n") ;
				}
					
			}			
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		 
	}	
	
	private void dispose()
	{
		linksMap = null ;
	}

}
